{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KineticsデータセットでECO用のDataLoaderを作成する\n",
    "\n",
    "本ファイルでは、Kineteicsの動画データを使い、ECO用のDataLoaderを作成します。\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 9.4 学習目標\n",
    "\n",
    "1.\tKinetics動画データセットをダウンロードできるようになる\n",
    "2.\t動画データをフレームごとの画像データに変換できるようになる\n",
    "3.\tECOで使用するためのDataLoaderを実装できるようになる\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 事前準備\n",
    "\n",
    "- 本書の内容に従い、Kineticsの動画のダウンロードと、動画データをframeごとに画像データに変換する操作を行ってください\n",
    "\n",
    "- 仮想環境pytorch_p36で実行します"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from PIL import Image\n",
    "import csv\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.utils.data\n",
    "from torch import nn\n",
    "\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 動画を画像データにしたフォルダへのファイルパスのリストを作成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/kinetics_videos/arm wrestling/C4lCVBZ3ux0_000028_000038\n",
      "./data/kinetics_videos/arm wrestling/ehLnj7pXnYE_000027_000037\n"
     ]
    }
   ],
   "source": [
    "def make_datapath_list(root_path):\n",
    "    \"\"\"\n",
    "    動画を画像データにしたフォルダへのファイルパスリストを作成する。\n",
    "    root_path : str、データフォルダへのrootパス\n",
    "    Returns：ret : video_list、動画を画像データにしたフォルダへのファイルパスリスト\n",
    "    \"\"\"\n",
    "\n",
    "    # 動画を画像データにしたフォルダへのファイルパスリスト\n",
    "    video_list = list()\n",
    "\n",
    "    # root_pathにある、クラスの種類とパスを取得\n",
    "    class_list = os.listdir(path=root_path)\n",
    "\n",
    "    # 各クラスの動画ファイルを画像化したフォルダへのパスを取得\n",
    "    for class_list_i in (class_list):  # クラスごとのループ\n",
    "\n",
    "        # クラスのフォルダへのパスを取得\n",
    "        class_path = os.path.join(root_path, class_list_i)\n",
    "\n",
    "        # 各クラスのフォルダ内の画像フォルダを取得するループ\n",
    "        for file_name in os.listdir(class_path):\n",
    "\n",
    "            # ファイル名と拡張子に分割\n",
    "            name, ext = os.path.splitext(file_name)\n",
    "\n",
    "            # フォルダでないmp4ファイルは無視\n",
    "            if ext == '.mp4':\n",
    "                continue\n",
    "\n",
    "            # 動画ファイルを画像に分割して保存したフォルダのパスを取得\n",
    "            video_img_directory_path = os.path.join(class_path, name)\n",
    "\n",
    "            # vieo_listに追加\n",
    "            video_list.append(video_img_directory_path)\n",
    "\n",
    "    return video_list\n",
    "\n",
    "\n",
    "# 動作確認\n",
    "root_path = './data/kinetics_videos/'\n",
    "video_list = make_datapath_list(root_path)\n",
    "print(video_list[0])\n",
    "print(video_list[1])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 動画の前処理クラスを作成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VideoTransform():\n",
    "    \"\"\"\n",
    "    動画を画像にした画像ファイルの前処理クラス。学習時と推論時で異なる動作をします。\n",
    "    動画を画像に分割しているため、分割された画像たちをまとめて前処理する点に注意してください。\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, resize, crop_size, mean, std):\n",
    "        self.data_transform = {\n",
    "            'train': torchvision.transforms.Compose([\n",
    "                # DataAugumentation()  # 今回は省略\n",
    "                GroupResize(int(resize)),  # 画像をまとめてリサイズ　\n",
    "                GroupCenterCrop(crop_size),  # 画像をまとめてセンタークロップ\n",
    "                GroupToTensor(),  # データをPyTorchのテンソルに\n",
    "                GroupImgNormalize(mean, std),  # データを標準化\n",
    "                Stack()  # 複数画像をframes次元で結合させる\n",
    "            ]),\n",
    "            'val': torchvision.transforms.Compose([\n",
    "                GroupResize(int(resize)),  # 画像をまとめてリサイズ　\n",
    "                GroupCenterCrop(crop_size),  # 画像をまとめてセンタークロップ\n",
    "                GroupToTensor(),  # データをPyTorchのテンソルに\n",
    "                GroupImgNormalize(mean, std),  # データを標準化\n",
    "                Stack()  # 複数画像をframes次元で結合させる\n",
    "            ])\n",
    "        }\n",
    "\n",
    "    def __call__(self, img_group, phase):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        phase : 'train' or 'val'\n",
    "            前処理のモードを指定。\n",
    "        \"\"\"\n",
    "        return self.data_transform[phase](img_group)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 前処理で使用するクラスたちの定義\n",
    "\n",
    "\n",
    "class GroupResize():\n",
    "    ''' 画像をまとめてリスケールするクラス。\n",
    "    画像の短い方の辺の長さがresizeに変換される。\n",
    "    アスペクト比は保たれる。\n",
    "    '''\n",
    "\n",
    "    def __init__(self, resize, interpolation=Image.BILINEAR):\n",
    "        '''リスケールする処理を用意'''\n",
    "        self.rescaler = torchvision.transforms.Resize(resize, interpolation)\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        '''リスケールをimg_group(リスト)内の各imgに実施'''\n",
    "        return [self.rescaler(img) for img in img_group]\n",
    "\n",
    "\n",
    "class GroupCenterCrop():\n",
    "    ''' 画像をまとめてセンタークロップするクラス。\n",
    "        （crop_size, crop_size）の画像を切り出す。\n",
    "    '''\n",
    "\n",
    "    def __init__(self, crop_size):\n",
    "        '''センタークロップする処理を用意'''\n",
    "        self.ccrop = torchvision.transforms.CenterCrop(crop_size)\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        '''センタークロップをimg_group(リスト)内の各imgに実施'''\n",
    "        return [self.ccrop(img) for img in img_group]\n",
    "\n",
    "\n",
    "class GroupToTensor():\n",
    "    ''' 画像をまとめてテンソル化するクラス。\n",
    "    '''\n",
    "\n",
    "    def __init__(self):\n",
    "        '''テンソル化する処理を用意'''\n",
    "        self.to_tensor = torchvision.transforms.ToTensor()\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        '''テンソル化をimg_group(リスト)内の各imgに実施\n",
    "        0から1ではなく、0から255で扱うため、255をかけ算する。\n",
    "        0から255で扱うのは、学習済みデータの形式に合わせるため\n",
    "        '''\n",
    "\n",
    "        return [self.to_tensor(img)*255 for img in img_group]\n",
    "\n",
    "\n",
    "class GroupImgNormalize():\n",
    "    ''' 画像をまとめて標準化するクラス。\n",
    "    '''\n",
    "\n",
    "    def __init__(self, mean, std):\n",
    "        '''標準化する処理を用意'''\n",
    "        self.normlize = torchvision.transforms.Normalize(mean, std)\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        '''標準化をimg_group(リスト)内の各imgに実施'''\n",
    "        return [self.normlize(img) for img in img_group]\n",
    "\n",
    "\n",
    "class Stack():\n",
    "    ''' 画像を一つのテンソルにまとめるクラス。\n",
    "    '''\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        '''img_groupはtorch.Size([3, 224, 224])を要素とするリスト\n",
    "        '''\n",
    "        ret = torch.cat([(x.flip(dims=[0])).unsqueeze(dim=0)\n",
    "                         for x in img_group], dim=0)  # frames次元で結合\n",
    "        # x.flip(dims=[0])は色チャネルをRGBからBGRへと順番を変えています（元の学習データがBGRであったため）\n",
    "        # unsqueeze(dim=0)はあらたにframes用の次元を作成しています\n",
    "\n",
    "        return ret\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasetの作成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'abseiling': 0,\n",
       " 'air drumming': 1,\n",
       " 'answering questions': 2,\n",
       " 'applauding': 3,\n",
       " 'applying cream': 4,\n",
       " 'archery': 5,\n",
       " 'arm wrestling': 6,\n",
       " 'arranging flowers': 7,\n",
       " 'assembling computer': 8,\n",
       " 'auctioning': 9,\n",
       " 'baby waking up': 10,\n",
       " 'baking cookies': 11,\n",
       " 'balloon blowing': 12,\n",
       " 'bandaging': 13,\n",
       " 'barbequing': 14,\n",
       " 'bartending': 15,\n",
       " 'beatboxing': 16,\n",
       " 'bee keeping': 17,\n",
       " 'belly dancing': 18,\n",
       " 'bench pressing': 19,\n",
       " 'bending back': 20,\n",
       " 'bending metal': 21,\n",
       " 'biking through snow': 22,\n",
       " 'blasting sand': 23,\n",
       " 'blowing glass': 24,\n",
       " 'blowing leaves': 25,\n",
       " 'blowing nose': 26,\n",
       " 'blowing out candles': 27,\n",
       " 'bobsledding': 28,\n",
       " 'bookbinding': 29,\n",
       " 'bouncing on trampoline': 30,\n",
       " 'bowling': 31,\n",
       " 'braiding hair': 32,\n",
       " 'breading or breadcrumbing': 33,\n",
       " 'breakdancing': 34,\n",
       " 'brush painting': 35,\n",
       " 'brushing hair': 36,\n",
       " 'brushing teeth': 37,\n",
       " 'building cabinet': 38,\n",
       " 'building shed': 39,\n",
       " 'bungee jumping': 40,\n",
       " 'busking': 41,\n",
       " 'canoeing or kayaking': 42,\n",
       " 'capoeira': 43,\n",
       " 'carrying baby': 44,\n",
       " 'cartwheeling': 45,\n",
       " 'carving pumpkin': 46,\n",
       " 'catching fish': 47,\n",
       " 'catching or throwing baseball': 48,\n",
       " 'catching or throwing frisbee': 49,\n",
       " 'catching or throwing softball': 50,\n",
       " 'celebrating': 51,\n",
       " 'changing oil': 52,\n",
       " 'changing wheel': 53,\n",
       " 'checking tires': 54,\n",
       " 'cheerleading': 55,\n",
       " 'chopping wood': 56,\n",
       " 'clapping': 57,\n",
       " 'clay pottery making': 58,\n",
       " 'clean and jerk': 59,\n",
       " 'cleaning floor': 60,\n",
       " 'cleaning gutters': 61,\n",
       " 'cleaning pool': 62,\n",
       " 'cleaning shoes': 63,\n",
       " 'cleaning toilet': 64,\n",
       " 'cleaning windows': 65,\n",
       " 'climbing a rope': 66,\n",
       " 'climbing ladder': 67,\n",
       " 'climbing tree': 68,\n",
       " 'contact juggling': 69,\n",
       " 'cooking chicken': 70,\n",
       " 'cooking egg': 71,\n",
       " 'cooking on campfire': 72,\n",
       " 'cooking sausages': 73,\n",
       " 'counting money': 74,\n",
       " 'country line dancing': 75,\n",
       " 'cracking neck': 76,\n",
       " 'crawling baby': 77,\n",
       " 'crossing river': 78,\n",
       " 'crying': 79,\n",
       " 'curling hair': 80,\n",
       " 'cutting nails': 81,\n",
       " 'cutting pineapple': 82,\n",
       " 'cutting watermelon': 83,\n",
       " 'dancing ballet': 84,\n",
       " 'dancing charleston': 85,\n",
       " 'dancing gangnam style': 86,\n",
       " 'dancing macarena': 87,\n",
       " 'deadlifting': 88,\n",
       " 'decorating the christmas tree': 89,\n",
       " 'digging': 90,\n",
       " 'dining': 91,\n",
       " 'disc golfing': 92,\n",
       " 'diving cliff': 93,\n",
       " 'dodgeball': 94,\n",
       " 'doing aerobics': 95,\n",
       " 'doing laundry': 96,\n",
       " 'doing nails': 97,\n",
       " 'drawing': 98,\n",
       " 'dribbling basketball': 99,\n",
       " 'drinking': 100,\n",
       " 'drinking beer': 101,\n",
       " 'drinking shots': 102,\n",
       " 'driving car': 103,\n",
       " 'driving tractor': 104,\n",
       " 'drop kicking': 105,\n",
       " 'drumming fingers': 106,\n",
       " 'dunking basketball': 107,\n",
       " 'dying hair': 108,\n",
       " 'eating burger': 109,\n",
       " 'eating cake': 110,\n",
       " 'eating carrots': 111,\n",
       " 'eating chips': 112,\n",
       " 'eating doughnuts': 113,\n",
       " 'eating hotdog': 114,\n",
       " 'eating ice cream': 115,\n",
       " 'eating spaghetti': 116,\n",
       " 'eating watermelon': 117,\n",
       " 'egg hunting': 118,\n",
       " 'exercising arm': 119,\n",
       " 'exercising with an exercise ball': 120,\n",
       " 'extinguishing fire': 121,\n",
       " 'faceplanting': 122,\n",
       " 'feeding birds': 123,\n",
       " 'feeding fish': 124,\n",
       " 'feeding goats': 125,\n",
       " 'filling eyebrows': 126,\n",
       " 'finger snapping': 127,\n",
       " 'fixing hair': 128,\n",
       " 'flipping pancake': 129,\n",
       " 'flying kite': 130,\n",
       " 'folding clothes': 131,\n",
       " 'folding napkins': 132,\n",
       " 'folding paper': 133,\n",
       " 'front raises': 134,\n",
       " 'frying vegetables': 135,\n",
       " 'garbage collecting': 136,\n",
       " 'gargling': 137,\n",
       " 'getting a haircut': 138,\n",
       " 'getting a tattoo': 139,\n",
       " 'giving or receiving award': 140,\n",
       " 'golf chipping': 141,\n",
       " 'golf driving': 142,\n",
       " 'golf putting': 143,\n",
       " 'grinding meat': 144,\n",
       " 'grooming dog': 145,\n",
       " 'grooming horse': 146,\n",
       " 'gymnastics tumbling': 147,\n",
       " 'hammer throw': 148,\n",
       " 'headbanging': 149,\n",
       " 'headbutting': 150,\n",
       " 'high jump': 151,\n",
       " 'high kick': 152,\n",
       " 'hitting baseball': 153,\n",
       " 'hockey stop': 154,\n",
       " 'holding snake': 155,\n",
       " 'hopscotch': 156,\n",
       " 'hoverboarding': 157,\n",
       " 'hugging': 158,\n",
       " 'hula hooping': 159,\n",
       " 'hurdling': 160,\n",
       " 'hurling (sport)': 161,\n",
       " 'ice climbing': 162,\n",
       " 'ice fishing': 163,\n",
       " 'ice skating': 164,\n",
       " 'ironing': 165,\n",
       " 'javelin throw': 166,\n",
       " 'jetskiing': 167,\n",
       " 'jogging': 168,\n",
       " 'juggling balls': 169,\n",
       " 'juggling fire': 170,\n",
       " 'juggling soccer ball': 171,\n",
       " 'jumping into pool': 172,\n",
       " 'jumpstyle dancing': 173,\n",
       " 'kicking field goal': 174,\n",
       " 'kicking soccer ball': 175,\n",
       " 'kissing': 176,\n",
       " 'kitesurfing': 177,\n",
       " 'knitting': 178,\n",
       " 'krumping': 179,\n",
       " 'laughing': 180,\n",
       " 'laying bricks': 181,\n",
       " 'long jump': 182,\n",
       " 'lunge': 183,\n",
       " 'making a cake': 184,\n",
       " 'making a sandwich': 185,\n",
       " 'making bed': 186,\n",
       " 'making jewelry': 187,\n",
       " 'making pizza': 188,\n",
       " 'making snowman': 189,\n",
       " 'making sushi': 190,\n",
       " 'making tea': 191,\n",
       " 'marching': 192,\n",
       " 'massaging back': 193,\n",
       " 'massaging feet': 194,\n",
       " 'massaging legs': 195,\n",
       " \"massaging person's head\": 196,\n",
       " 'milking cow': 197,\n",
       " 'mopping floor': 198,\n",
       " 'motorcycling': 199,\n",
       " 'moving furniture': 200,\n",
       " 'mowing lawn': 201,\n",
       " 'news anchoring': 202,\n",
       " 'opening bottle': 203,\n",
       " 'opening present': 204,\n",
       " 'paragliding': 205,\n",
       " 'parasailing': 206,\n",
       " 'parkour': 207,\n",
       " 'passing American football (in game)': 208,\n",
       " 'passing American football (not in game)': 209,\n",
       " 'peeling apples': 210,\n",
       " 'peeling potatoes': 211,\n",
       " 'petting animal (not cat)': 212,\n",
       " 'petting cat': 213,\n",
       " 'picking fruit': 214,\n",
       " 'planting trees': 215,\n",
       " 'plastering': 216,\n",
       " 'playing accordion': 217,\n",
       " 'playing badminton': 218,\n",
       " 'playing bagpipes': 219,\n",
       " 'playing basketball': 220,\n",
       " 'playing bass guitar': 221,\n",
       " 'playing cards': 222,\n",
       " 'playing cello': 223,\n",
       " 'playing chess': 224,\n",
       " 'playing clarinet': 225,\n",
       " 'playing controller': 226,\n",
       " 'playing cricket': 227,\n",
       " 'playing cymbals': 228,\n",
       " 'playing didgeridoo': 229,\n",
       " 'playing drums': 230,\n",
       " 'playing flute': 231,\n",
       " 'playing guitar': 232,\n",
       " 'playing harmonica': 233,\n",
       " 'playing harp': 234,\n",
       " 'playing ice hockey': 235,\n",
       " 'playing keyboard': 236,\n",
       " 'playing kickball': 237,\n",
       " 'playing monopoly': 238,\n",
       " 'playing organ': 239,\n",
       " 'playing paintball': 240,\n",
       " 'playing piano': 241,\n",
       " 'playing poker': 242,\n",
       " 'playing recorder': 243,\n",
       " 'playing saxophone': 244,\n",
       " 'playing squash or racquetball': 245,\n",
       " 'playing tennis': 246,\n",
       " 'playing trombone': 247,\n",
       " 'playing trumpet': 248,\n",
       " 'playing ukulele': 249,\n",
       " 'playing violin': 250,\n",
       " 'playing volleyball': 251,\n",
       " 'playing xylophone': 252,\n",
       " 'pole vault': 253,\n",
       " 'presenting weather forecast': 254,\n",
       " 'pull ups': 255,\n",
       " 'pumping fist': 256,\n",
       " 'pumping gas': 257,\n",
       " 'punching bag': 258,\n",
       " 'punching person (boxing)': 259,\n",
       " 'push up': 260,\n",
       " 'pushing car': 261,\n",
       " 'pushing cart': 262,\n",
       " 'pushing wheelchair': 263,\n",
       " 'reading book': 264,\n",
       " 'reading newspaper': 265,\n",
       " 'recording music': 266,\n",
       " 'riding a bike': 267,\n",
       " 'riding camel': 268,\n",
       " 'riding elephant': 269,\n",
       " 'riding mechanical bull': 270,\n",
       " 'riding mountain bike': 271,\n",
       " 'riding mule': 272,\n",
       " 'riding or walking with horse': 273,\n",
       " 'riding scooter': 274,\n",
       " 'riding unicycle': 275,\n",
       " 'ripping paper': 276,\n",
       " 'robot dancing': 277,\n",
       " 'rock climbing': 278,\n",
       " 'rock scissors paper': 279,\n",
       " 'roller skating': 280,\n",
       " 'running on treadmill': 281,\n",
       " 'sailing': 282,\n",
       " 'salsa dancing': 283,\n",
       " 'sanding floor': 284,\n",
       " 'scrambling eggs': 285,\n",
       " 'scuba diving': 286,\n",
       " 'setting table': 287,\n",
       " 'shaking hands': 288,\n",
       " 'shaking head': 289,\n",
       " 'sharpening knives': 290,\n",
       " 'sharpening pencil': 291,\n",
       " 'shaving head': 292,\n",
       " 'shaving legs': 293,\n",
       " 'shearing sheep': 294,\n",
       " 'shining shoes': 295,\n",
       " 'shooting basketball': 296,\n",
       " 'shooting goal (soccer)': 297,\n",
       " 'shot put': 298,\n",
       " 'shoveling snow': 299,\n",
       " 'shredding paper': 300,\n",
       " 'shuffling cards': 301,\n",
       " 'side kick': 302,\n",
       " 'sign language interpreting': 303,\n",
       " 'singing': 304,\n",
       " 'situp': 305,\n",
       " 'skateboarding': 306,\n",
       " 'ski jumping': 307,\n",
       " 'skiing (not slalom or crosscountry)': 308,\n",
       " 'skiing crosscountry': 309,\n",
       " 'skiing slalom': 310,\n",
       " 'skipping rope': 311,\n",
       " 'skydiving': 312,\n",
       " 'slacklining': 313,\n",
       " 'slapping': 314,\n",
       " 'sled dog racing': 315,\n",
       " 'smoking': 316,\n",
       " 'smoking hookah': 317,\n",
       " 'snatch weight lifting': 318,\n",
       " 'sneezing': 319,\n",
       " 'sniffing': 320,\n",
       " 'snorkeling': 321,\n",
       " 'snowboarding': 322,\n",
       " 'snowkiting': 323,\n",
       " 'snowmobiling': 324,\n",
       " 'somersaulting': 325,\n",
       " 'spinning poi': 326,\n",
       " 'spray painting': 327,\n",
       " 'spraying': 328,\n",
       " 'springboard diving': 329,\n",
       " 'squat': 330,\n",
       " 'sticking tongue out': 331,\n",
       " 'stomping grapes': 332,\n",
       " 'stretching arm': 333,\n",
       " 'stretching leg': 334,\n",
       " 'strumming guitar': 335,\n",
       " 'surfing crowd': 336,\n",
       " 'surfing water': 337,\n",
       " 'sweeping floor': 338,\n",
       " 'swimming backstroke': 339,\n",
       " 'swimming breast stroke': 340,\n",
       " 'swimming butterfly stroke': 341,\n",
       " 'swing dancing': 342,\n",
       " 'swinging legs': 343,\n",
       " 'swinging on something': 344,\n",
       " 'sword fighting': 345,\n",
       " 'tai chi': 346,\n",
       " 'taking a shower': 347,\n",
       " 'tango dancing': 348,\n",
       " 'tap dancing': 349,\n",
       " 'tapping guitar': 350,\n",
       " 'tapping pen': 351,\n",
       " 'tasting beer': 352,\n",
       " 'tasting food': 353,\n",
       " 'testifying': 354,\n",
       " 'texting': 355,\n",
       " 'throwing axe': 356,\n",
       " 'throwing ball': 357,\n",
       " 'throwing discus': 358,\n",
       " 'tickling': 359,\n",
       " 'tobogganing': 360,\n",
       " 'tossing coin': 361,\n",
       " 'tossing salad': 362,\n",
       " 'training dog': 363,\n",
       " 'trapezing': 364,\n",
       " 'trimming or shaving beard': 365,\n",
       " 'trimming trees': 366,\n",
       " 'triple jump': 367,\n",
       " 'tying bow tie': 368,\n",
       " 'tying knot (not on a tie)': 369,\n",
       " 'tying tie': 370,\n",
       " 'unboxing': 371,\n",
       " 'unloading truck': 372,\n",
       " 'using computer': 373,\n",
       " 'using remote controller (not gaming)': 374,\n",
       " 'using segway': 375,\n",
       " 'vault': 376,\n",
       " 'waiting in line': 377,\n",
       " 'walking the dog': 378,\n",
       " 'washing dishes': 379,\n",
       " 'washing feet': 380,\n",
       " 'washing hair': 381,\n",
       " 'washing hands': 382,\n",
       " 'water skiing': 383,\n",
       " 'water sliding': 384,\n",
       " 'watering plants': 385,\n",
       " 'waxing back': 386,\n",
       " 'waxing chest': 387,\n",
       " 'waxing eyebrows': 388,\n",
       " 'waxing legs': 389,\n",
       " 'weaving basket': 390,\n",
       " 'welding': 391,\n",
       " 'whistling': 392,\n",
       " 'windsurfing': 393,\n",
       " 'wrapping present': 394,\n",
       " 'wrestling': 395,\n",
       " 'writing': 396,\n",
       " 'yawning': 397,\n",
       " 'yoga': 398,\n",
       " 'zumba': 399}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Kinetics-400のラベル名をIDに変換する辞書と、逆にIDをラベル名に変換する辞書を用意\n",
    "\n",
    "\n",
    "def get_label_id_dictionary(label_dicitionary_path='./video_download/kinetics_400_label_dicitionary.csv'):\n",
    "    label_id_dict = {}\n",
    "    id_label_dict = {}\n",
    "\n",
    "    with open(label_dicitionary_path, encoding=\"utf-8_sig\") as f:\n",
    "\n",
    "        # 読み込む\n",
    "        reader = csv.DictReader(f, delimiter=\",\", quotechar='\"')\n",
    "\n",
    "        # 1行ずつ読み込み、辞書型変数に追加します\n",
    "        for row in reader:\n",
    "            label_id_dict.setdefault(\n",
    "                row[\"class_label\"], int(row[\"label_id\"])-1)\n",
    "            id_label_dict.setdefault(\n",
    "                int(row[\"label_id\"])-1, row[\"class_label\"])\n",
    "\n",
    "    return label_id_dict,  id_label_dict\n",
    "\n",
    "\n",
    "# 確認\n",
    "label_dicitionary_path = './video_download/kinetics_400_label_dicitionary.csv'\n",
    "label_id_dict, id_label_dict = get_label_id_dictionary(label_dicitionary_path)\n",
    "label_id_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VideoDataset(torch.utils.data.Dataset):\n",
    "    \"\"\"\n",
    "    動画のDataset\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, video_list, label_id_dict, num_segments, phase, transform, img_tmpl='image_{:05d}.jpg'):\n",
    "        self.video_list = video_list  # 動画画像のフォルダへのパスリスト\n",
    "        self.label_id_dict = label_id_dict  # ラベル名をidに変換する辞書型変数\n",
    "        self.num_segments = num_segments  # 動画を何分割して使用するのかを決める\n",
    "        self.phase = phase  # train or val\n",
    "        self.transform = transform  # 前処理\n",
    "        self.img_tmpl = img_tmpl  # 読み込みたい画像のファイル名のテンプレート\n",
    "\n",
    "    def __len__(self):\n",
    "        '''動画の数を返す'''\n",
    "        return len(self.video_list)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        '''\n",
    "        前処理をした画像たちのデータとラベル、ラベルIDを取得\n",
    "        '''\n",
    "        imgs_transformed, label, label_id, dir_path = self.pull_item(index)\n",
    "        return imgs_transformed, label, label_id, dir_path\n",
    "\n",
    "    def pull_item(self, index):\n",
    "        '''前処理をした画像たちのデータとラベル、ラベルIDを取得'''\n",
    "\n",
    "        # 1. 画像たちをリストに読み込む\n",
    "        dir_path = self.video_list[index]  # 画像が格納されたフォルダ\n",
    "        indices = self._get_indices(dir_path)  # 読み込む画像idxを求める\n",
    "        img_group = self._load_imgs(\n",
    "            dir_path, self.img_tmpl, indices)  # リストに読み込む\n",
    "\n",
    "        # 2. ラベルの取得し、idに変換する\n",
    "        label = (dir_path.split('/')[3].split('/')[0])\n",
    "        label_id = self.label_id_dict[label] # idを取得\n",
    "\n",
    "        # 3. 前処理を実施\n",
    "        imgs_transformed = self.transform(img_group, phase=self.phase)\n",
    "\n",
    "        return imgs_transformed, label, label_id, dir_path\n",
    "\n",
    "    def _load_imgs(self, dir_path, img_tmpl, indices):\n",
    "        '''画像をまとめて読み込み、リスト化する関数'''\n",
    "        img_group = []  # 画像を格納するリスト\n",
    "\n",
    "        for idx in indices:\n",
    "            # 画像のパスを取得\n",
    "            file_path = os.path.join(dir_path, img_tmpl.format(idx))\n",
    "\n",
    "            # 画像を読み込む\n",
    "            img = Image.open(file_path).convert('RGB')\n",
    "\n",
    "            # リストに追加\n",
    "            img_group.append(img)\n",
    "        return img_group\n",
    "\n",
    "    def _get_indices(self, dir_path):\n",
    "        \"\"\"\n",
    "        動画全体をself.num_segmentに分割した際に取得する動画のidxのリストを取得する\n",
    "        \"\"\"\n",
    "        # 動画のフレーム数を求める\n",
    "        file_list = os.listdir(path=dir_path)\n",
    "        num_frames = len(file_list)\n",
    "\n",
    "        # 動画の取得間隔幅を求める\n",
    "        tick = (num_frames) / float(self.num_segments)\n",
    "        # 250 / 16 = 15.625\n",
    "        # 動画の取得間隔幅で取り出す際のidxをリストで求める\n",
    "        indices = np.array([int(tick / 2.0 + tick * x)\n",
    "                            for x in range(self.num_segments)])+1\n",
    "        # 250frameで16frame抽出の場合\n",
    "        # indices = [  8  24  40  55  71  86 102 118 133 149 165 180 196 211 227 243]\n",
    "\n",
    "        return indices\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 3, 224, 224])\n",
      "arm wrestling\n",
      "6\n",
      "./data/kinetics_videos/arm wrestling/C4lCVBZ3ux0_000028_000038\n"
     ]
    }
   ],
   "source": [
    "# 動作確認\n",
    "\n",
    "# vieo_listの作成\n",
    "root_path = './data/kinetics_videos/'\n",
    "video_list = make_datapath_list(root_path)\n",
    "\n",
    "# 前処理の設定\n",
    "resize, crop_size = 224, 224\n",
    "mean, std = [104, 117, 123], [1, 1, 1]\n",
    "video_transform = VideoTransform(resize, crop_size, mean, std)\n",
    "\n",
    "# Datasetの作成\n",
    "# num_segments は 動画を何分割して使用するのかを決める\n",
    "val_dataset = VideoDataset(video_list, label_id_dict, num_segments=16,\n",
    "                           phase=\"val\", transform=video_transform, img_tmpl='image_{:05d}.jpg')\n",
    "\n",
    "# データの取り出し例\n",
    "# 出力は、imgs_transformed, label, label_id, dir_path\n",
    "index = 0\n",
    "print(val_dataset.__getitem__(index)[0].shape)  # 画像たちのテンソル\n",
    "print(val_dataset.__getitem__(index)[1])  # ラベル名\n",
    "print(val_dataset.__getitem__(index)[2])  # ラベルID\n",
    "print(val_dataset.__getitem__(index)[3])  # 動画へのパス\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 16, 3, 224, 224])\n"
     ]
    }
   ],
   "source": [
    "# DataLoaderにします\n",
    "batch_size = 8\n",
    "val_dataloader = torch.utils.data.DataLoader(\n",
    "    val_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "# 動作確認\n",
    "batch_iterator = iter(val_dataloader)  # イテレータに変換\n",
    "imgs_transformeds, labels, label_ids, dir_path = next(\n",
    "    batch_iterator)  # 1番目の要素を取り出す\n",
    "print(imgs_transformeds.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以上"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
